package util;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.URL;
import java.net.HttpURLConnection;

import crawler.CrawlerDocument;

public class FetchUtil {

	public static void fetch(CrawlerDocument doc, String encoding) throws IOException  {
		URL url = doc.getUrl();
		HttpURLConnection conn = (HttpURLConnection)url.openConnection();
		conn.setRequestProperty("User-Agent", "Mozilla/5.0 (X11; Linux i686) AppleWebKit/534.24 (KHTML, like Gecko) Chrome/11.0.696.71 Safari/534.24");
		BufferedReader reader = new BufferedReader(
				new InputStreamReader(conn.getInputStream(), encoding));
		String content = "";
		String line = null;
		while((line = reader.readLine()) != null){
			content += line + "\r\n";
		}
		conn.disconnect();
		doc.addFeature("content", content);
	}
	
	public static void fetch(CrawlerDocument doc) throws IOException {
		fetch(doc, "utf-8");
	}
}
